{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "18ow18aTZqRL"
      },
      "outputs": [],
      "source": [
        "!pip install flash-attn --no-build-isolation\n",
        "!pip install tiktoken\n",
        "!pip install datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hBhJzS-r4oNn"
      },
      "outputs": [],
      "source": [
        "!rm -rf /content/baike_chunk_512_5.6M_0.parquet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "od2KiXgPnAcC"
      },
      "outputs": [],
      "source": [
        "import sys\n",
        "\n",
        "# 将某个路径添加到系统路径\n",
        "sys.path.append('/content/Model')\n",
        "\n",
        "# 然后就可以从该路径导入模块\n",
        "# import your_module"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HneH4fTvmICs"
      },
      "outputs": [],
      "source": [
        "globals().clear()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3DtCmZu2Ynxe"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "from transformers import (\n",
        "    DataCollatorForLanguageModeling,\n",
        "    PreTrainedTokenizerFast,\n",
        "    Trainer,\n",
        "    TrainerCallback,\n",
        "    TrainingArguments,\n",
        ")\n",
        "from transformers.trainer_callback import TrainerControl, TrainerState\n",
        "\n",
        "from arguments import PretrainArguments\n",
        "from datasets import Dataset, load_dataset\n",
        "from qwen.configuration_qwen import QWenConfig\n",
        "from qwen.modeling_qwen import QWenLMHeadModel\n",
        "from qwen.tokenization_qwen import QWenTokenizer\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TV_yILs-ZptM",
        "outputId": "be99a0d9-7986-4a4f-8090-2dc4cd74afc0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "final vocab size: 151872\n"
          ]
        }
      ],
      "source": [
        "pretrain_args = PretrainArguments()\n",
        "tokenizer = QWenTokenizer.from_pretrained(pretrain_args.tokenizer_dir)\n",
        "tokenizer.pad_token_id = tokenizer.im_end_id\n",
        "vocab_size = len(tokenizer)\n",
        "if vocab_size % 64 != 0:\n",
        "    vocab_size = (vocab_size // 64 + 1) * 64\n",
        "print(f\"final vocab size: {vocab_size}\")\n",
        "map_dtype = np.uint16 if vocab_size < 65535 else np.uint32"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "x1FClWjhgxiI"
      },
      "outputs": [],
      "source": [
        "def token_to_id(samples: dict) -> dict:\n",
        "\n",
        "    batch_txt = samples[\"text\"]\n",
        "    outputs = tokenizer(\n",
        "        batch_txt,\n",
        "        padding=False,\n",
        "        return_attention_mask=False,\n",
        "        truncation=True,\n",
        "        max_length=pretrain_args.max_seq_len\n",
        "    )\n",
        "\n",
        "    input_ids = [np.array(item, dtype=map_dtype) for item in outputs[\"input_ids\"]]\n",
        "\n",
        "    return {\"input_ids\": input_ids}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "20qz1wsZg41x"
      },
      "outputs": [],
      "source": [
        "def get_maped_dataset(files) -> Dataset:\n",
        "    dataset = load_dataset(\n",
        "        path=\"/content/Model/datasets\",\n",
        "        data_files=files,\n",
        "        split=\"train\",\n",
        "        cache_dir=\".cache\",\n",
        "        keep_in_memory=False,\n",
        "    )\n",
        "    maped_dataset = dataset.map(\n",
        "        token_to_id,\n",
        "        batched=True,\n",
        "        batch_size=10000,\n",
        "        remove_columns=dataset.column_names,\n",
        "        num_proc=24,\n",
        "        keep_in_memory=False,\n",
        "    )\n",
        "    return maped_dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 266,
          "referenced_widgets": [
            "8c1ded6dd5574e19aa8c88e0be2164e9",
            "9c994f6fa207414bb46441f3d46b258a",
            "9d0f9a569f2844e7bd69790ad8b61ca7",
            "2b91a9a31b6646198bb9dbf08325df27",
            "1e6f10f4a78d4406984678b9c73cca48",
            "71d9196cabd74de2aa89344d641e5bba",
            "c65550c07c0c42b0a983ff1f05721430",
            "a3aafb8172d5495687c527649272d1dc",
            "61e046dd807a4243a6022ada12942a77",
            "6df60d2cb4ce46779d9293592730e54f",
            "09ab387176734fef99ab2bb871a03561",
            "2f733805302d472e8adb54b621ea12fd",
            "e29ce04398f1496c856062c355a4555d",
            "f166787b613f4a4ab0f7b1e38e3e153e",
            "dc63be992bf54c75bd49a19472723a89",
            "df5f7850ad134cc2905faf9af63074c0",
            "85edd6b782084728bc5ca6512b8cd7a8",
            "3b805a8f92e84d739357e8cc54f4deba",
            "22f09c483a434a35832f1a0a0e20a2a9",
            "10d88eb79fcd4fffa27d3f40ebe8ac23",
            "103cf81c5cbd41d8a0051ad1fe6d151d",
            "870238d863004b1ca038d51daae50e84",
            "0da60a62a0794f789ecf14e6c051abb7",
            "9522f14f4bd14b548db0cefcceef2fa3",
            "c6216f0615814f92ba52a5da45e0f28d",
            "aaa1bc8a499e41abae85b86ceb29b66b",
            "a9843f89c1054ec4adc226fdcdb1aba4",
            "1043d11e539742bfbb1698649843e8f0",
            "c41185e10467436abd9a100d0e797511",
            "f6edd1ad7deb4ac3b66e331117acefbb",
            "6b945880de87407fb4f279613b007654",
            "d0d5d38ac0344244b0e688102ee8b13a",
            "851855ecf5fb44f39d1e79162d5351f0",
            "caf7fa7133774ce3877d1c1b1b729f66",
            "ef2c5cd0f4d84a68bf4ca6a88f40d9ed",
            "9bcde6b4a56a467da9ea4056f36fbfbe",
            "84f0d5fa529543ed84627e79ea0ef49d",
            "1d5c753460984112ab52a18c1e470053",
            "7a5e77e55f7941e6a59d26f4f2371c03",
            "13471ff80cdc4ef18e360b8534afe0b7",
            "7361f038191d4617b69395850e8de8bb",
            "afc6e86bf48d4bff96eea6567a82d447",
            "46ea009434b04f20b1914525678ed04d",
            "d1f38e16bca04d528d84a2a05ce56312"
          ]
        },
        "id": "5qImyDzqg7JE",
        "outputId": "91d6a17f-40bc-4ff7-dcb4-9c55c41b7195"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8c1ded6dd5574e19aa8c88e0be2164e9",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2f733805302d472e8adb54b621ea12fd",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Map (num_proc=24):   0%|          | 0/6001888 [00:00<?, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0da60a62a0794f789ecf14e6c051abb7",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Generating train split: 0 examples [00:00, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "caf7fa7133774ce3877d1c1b1b729f66",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Map (num_proc=24):   0%|          | 0/8848 [00:00<?, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Dataset({\n",
            "    features: ['input_ids'],\n",
            "    num_rows: 6001888\n",
            "}) Dataset({\n",
            "    features: ['input_ids'],\n",
            "    num_rows: 8848\n",
            "})\n"
          ]
        }
      ],
      "source": [
        "train_dataset = get_maped_dataset(pretrain_args.train_files)\n",
        "eval_dataset = get_maped_dataset(pretrain_args.eval_file)\n",
        "print(train_dataset, eval_dataset)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CxpI2J21EmQI"
      },
      "outputs": [],
      "source": [
        "attn_implementation = \"flash_attention_2\"\n",
        "try:\n",
        "    from flash_attn import flash_attn_func\n",
        "except Exception as e:\n",
        "    attn_implementation = \"eager\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ciRDs-zMiB1n"
      },
      "outputs": [],
      "source": [
        "data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=False)\n",
        "if pretrain_args.attn_implementation == \"flash_attention_2\":\n",
        "    torch.set_default_dtype(torch.bfloat16)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fxS5Qt9NiJiG",
        "outputId": "6a658c1a-e3f1-4f11-a5f2-93d455a7462c"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:qwen.modeling_qwen:Warning: import flash_attn rms_norm fail, please install FlashAttention layer_norm to get higher efficiency https://github.com/Dao-AILab/flash-attention/tree/main/csrc/layer_norm\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "QWen size: 1432.0M parameters\n"
          ]
        }
      ],
      "source": [
        "config = QWenConfig.from_pretrained(\"/content/drive/MyDrive/Model/save/checkpoint-120000\")\n",
        "model = QWenLMHeadModel(config)\n",
        "\n",
        "model_size = sum(t.numel() for t in model.parameters())\n",
        "print(f\"QWen size: {model_size / 1000**2:.1f}M parameters\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v72VHkF_iLni"
      },
      "outputs": [],
      "source": [
        "class MyTrainerCallback(TrainerCallback):\n",
        "    log_cnt = 0\n",
        "\n",
        "    def on_log(\n",
        "        self,\n",
        "        args: TrainingArguments,\n",
        "        state: TrainerState,\n",
        "        control: TrainerControl,\n",
        "        **kwargs,\n",
        "    ):\n",
        "        \"\"\"\n",
        "        在打印 n 次日志后清除cuda缓存，适合低显存设备，能防止OOM\n",
        "        \"\"\"\n",
        "        self.log_cnt += 1\n",
        "        if self.log_cnt % 2 == 0:\n",
        "            torch.cuda.empty_cache()\n",
        "\n",
        "    def on_epoch_end(\n",
        "        self,\n",
        "        args: TrainingArguments,\n",
        "        state: TrainerState,\n",
        "        control: TrainerControl,\n",
        "        **kwargs,\n",
        "    ):\n",
        "        \"\"\"\n",
        "        在on_epoch_end时保存一次模型。\n",
        "        TrainingArguments的 save_strategy 中 epoch 和 steps 不兼容。要实现每隔 save_steps 步保存一次检查点，考虑到磁盘空间大小，最多只保存最近3个检查点。\n",
        "        \"\"\"\n",
        "        # 设置should_save=True并返回即可\n",
        "        control.should_save = True\n",
        "        return control\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OADQwvvqiOQH"
      },
      "outputs": [],
      "source": [
        "my_trainer_callback = MyTrainerCallback()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ju1JJmVIiQT3",
        "outputId": "764a9db3-5c8f-479a-cbe9-32d441045ff6"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.11/dist-packages/transformers/training_args.py:1575: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
            "  warnings.warn(\n"
          ]
        }
      ],
      "source": [
        "args = TrainingArguments(\n",
        "    output_dir=pretrain_args.model_save_dir,\n",
        "    per_device_train_batch_size=4,\n",
        "    per_device_eval_batch_size=4,\n",
        "    gradient_accumulation_steps=10,\n",
        "    num_train_epochs=1,\n",
        "    weight_decay=0.1,\n",
        "    ddp_find_unused_parameters=False,\n",
        "    warmup_steps=0,\n",
        "    learning_rate=1e-4,\n",
        "    evaluation_strategy=\"steps\",\n",
        "    eval_steps=20000,\n",
        "    save_steps=20000,\n",
        "    save_strategy=\"steps\",\n",
        "    save_total_limit=4,\n",
        "    report_to=\"tensorboard\",\n",
        "    optim=\"adamw_torch\",\n",
        "    lr_scheduler_type=\"cosine\",\n",
        "    bf16=True,\n",
        "    logging_steps=20,\n",
        "    log_level=\"info\",\n",
        "    logging_first_step=True,\n",
        "    # group_by_length=True,\n",
        "    # deepspeed='./ds_config_one_gpu.json',\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_muuCLjBiSZh",
        "outputId": "d778d186-f0e0-482c-c984-2ae5a5bda3f1"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "<ipython-input-14-9b3cf5b04614>:1: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
            "  trainer = Trainer(\n",
            "Using auto half precision backend\n"
          ]
        }
      ],
      "source": [
        "trainer = Trainer(\n",
        "    model=model,\n",
        "    tokenizer=tokenizer,\n",
        "    args=args,\n",
        "    data_collator=data_collator,\n",
        "    train_dataset=train_dataset,\n",
        "    eval_dataset=eval_dataset,\n",
        "    callbacks=[my_trainer_callback],\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "6NardXwLiUd5",
        "outputId": "d9fb77af-ff18-450c-a6a5-69bd3b8f35b5"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Loading model from /content/drive/MyDrive/Model/save/checkpoint-120000.\n",
            "***** Running training *****\n",
            "  Num examples = 6,001,888\n",
            "  Num Epochs = 1\n",
            "  Instantaneous batch size per device = 4\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 40\n",
            "  Gradient Accumulation steps = 10\n",
            "  Total optimization steps = 150,047\n",
            "  Number of trainable parameters = 1,431,996,416\n",
            "  Continuing training from checkpoint, will skip to saved global_step\n",
            "  Continuing training from epoch 0\n",
            "  Continuing training from global step 120000\n",
            "  Will skip the first 0 epochs then the first 1200000 batches in the first epoch.\n",
            "/usr/local/lib/python3.11/dist-packages/transformers/trainer.py:3105: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
            "  checkpoint_rng_state = torch.load(rng_file)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='150047' max='150047' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [150047/150047 9:20:21, Epoch 0/1]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>140000</td>\n",
              "      <td>2.782700</td>\n",
              "      <td>3.678705</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 8848\n",
            "  Batch size = 4\n",
            "Saving model checkpoint to /content/drive/MyDrive/Model/save/checkpoint-140000\n",
            "Configuration saved in /content/drive/MyDrive/Model/save/checkpoint-140000/config.json\n",
            "Configuration saved in /content/drive/MyDrive/Model/save/checkpoint-140000/generation_config.json\n",
            "Model weights saved in /content/drive/MyDrive/Model/save/checkpoint-140000/model.safetensors\n",
            "tokenizer config file saved in /content/drive/MyDrive/Model/save/checkpoint-140000/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/Model/save/checkpoint-140000/special_tokens_map.json\n",
            "Saving model checkpoint to /content/drive/MyDrive/Model/save/checkpoint-150047\n",
            "Configuration saved in /content/drive/MyDrive/Model/save/checkpoint-150047/config.json\n",
            "Configuration saved in /content/drive/MyDrive/Model/save/checkpoint-150047/generation_config.json\n",
            "Model weights saved in /content/drive/MyDrive/Model/save/checkpoint-150047/model.safetensors\n",
            "tokenizer config file saved in /content/drive/MyDrive/Model/save/checkpoint-150047/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/Model/save/checkpoint-150047/special_tokens_map.json\n",
            "Saving model checkpoint to /content/drive/MyDrive/Model/save/checkpoint-150047\n",
            "Configuration saved in /content/drive/MyDrive/Model/save/checkpoint-150047/config.json\n",
            "Configuration saved in /content/drive/MyDrive/Model/save/checkpoint-150047/generation_config.json\n",
            "Model weights saved in /content/drive/MyDrive/Model/save/checkpoint-150047/model.safetensors\n",
            "tokenizer config file saved in /content/drive/MyDrive/Model/save/checkpoint-150047/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/Model/save/checkpoint-150047/special_tokens_map.json\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 8848\n",
            "  Batch size = 4\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='442' max='2212' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [ 442/2212 00:18 < 01:13, 23.98 it/s]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='2212' max='2212' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [2212/2212 01:32]\n",
              "    </div>\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Saving model checkpoint to /content/drive/MyDrive/Model/save/\n",
            "Configuration saved in /content/drive/MyDrive/Model/save/config.json\n",
            "Configuration saved in /content/drive/MyDrive/Model/save/generation_config.json\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Perplexity: 37.46\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Model weights saved in /content/drive/MyDrive/Model/save/model.safetensors\n",
            "tokenizer config file saved in /content/drive/MyDrive/Model/save/tokenizer_config.json\n",
            "Special tokens file saved in /content/drive/MyDrive/Model/save/special_tokens_map.json\n"
          ]
        }
      ],
      "source": [
        "trainer.train(resume_from_checkpoint=True)\n",
        "\n",
        "#  计算困惑度Perplexity\n",
        "eval_results = trainer.evaluate()\n",
        "print(f\"Perplexity: {np.exp(eval_results['eval_loss']):.2f}\")\n",
        "\n",
        "\n",
        "# 最后保存训练的loss日志和模型\n",
        "trainer.save_model(pretrain_args.model_save_dir)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Q-rt2f8-qKFL"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "A100",
      "machine_shape": "hm",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "09ab387176734fef99ab2bb871a03561": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0da60a62a0794f789ecf14e6c051abb7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9522f14f4bd14b548db0cefcceef2fa3",
              "IPY_MODEL_c6216f0615814f92ba52a5da45e0f28d",
              "IPY_MODEL_aaa1bc8a499e41abae85b86ceb29b66b"
            ],
            "layout": "IPY_MODEL_a9843f89c1054ec4adc226fdcdb1aba4"
          }
        },
        "103cf81c5cbd41d8a0051ad1fe6d151d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1043d11e539742bfbb1698649843e8f0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "10d88eb79fcd4fffa27d3f40ebe8ac23": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "13471ff80cdc4ef18e360b8534afe0b7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1d5c753460984112ab52a18c1e470053": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1e6f10f4a78d4406984678b9c73cca48": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "22f09c483a434a35832f1a0a0e20a2a9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2b91a9a31b6646198bb9dbf08325df27": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6df60d2cb4ce46779d9293592730e54f",
            "placeholder": "​",
            "style": "IPY_MODEL_09ab387176734fef99ab2bb871a03561",
            "value": " 6001888/0 [00:21&lt;00:00, 657440.92 examples/s]"
          }
        },
        "2f733805302d472e8adb54b621ea12fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e29ce04398f1496c856062c355a4555d",
              "IPY_MODEL_f166787b613f4a4ab0f7b1e38e3e153e",
              "IPY_MODEL_dc63be992bf54c75bd49a19472723a89"
            ],
            "layout": "IPY_MODEL_df5f7850ad134cc2905faf9af63074c0"
          }
        },
        "3b805a8f92e84d739357e8cc54f4deba": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "46ea009434b04f20b1914525678ed04d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "61e046dd807a4243a6022ada12942a77": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6b945880de87407fb4f279613b007654": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6df60d2cb4ce46779d9293592730e54f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71d9196cabd74de2aa89344d641e5bba": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7361f038191d4617b69395850e8de8bb": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7a5e77e55f7941e6a59d26f4f2371c03": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "84f0d5fa529543ed84627e79ea0ef49d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_46ea009434b04f20b1914525678ed04d",
            "placeholder": "​",
            "style": "IPY_MODEL_d1f38e16bca04d528d84a2a05ce56312",
            "value": " 8848/8848 [00:21&lt;00:00, 423.36 examples/s]"
          }
        },
        "851855ecf5fb44f39d1e79162d5351f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "85edd6b782084728bc5ca6512b8cd7a8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "870238d863004b1ca038d51daae50e84": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8c1ded6dd5574e19aa8c88e0be2164e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9c994f6fa207414bb46441f3d46b258a",
              "IPY_MODEL_9d0f9a569f2844e7bd69790ad8b61ca7",
              "IPY_MODEL_2b91a9a31b6646198bb9dbf08325df27"
            ],
            "layout": "IPY_MODEL_1e6f10f4a78d4406984678b9c73cca48"
          }
        },
        "9522f14f4bd14b548db0cefcceef2fa3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1043d11e539742bfbb1698649843e8f0",
            "placeholder": "​",
            "style": "IPY_MODEL_c41185e10467436abd9a100d0e797511",
            "value": "Generating train split: "
          }
        },
        "9bcde6b4a56a467da9ea4056f36fbfbe": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7361f038191d4617b69395850e8de8bb",
            "max": 8848,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_afc6e86bf48d4bff96eea6567a82d447",
            "value": 8848
          }
        },
        "9c994f6fa207414bb46441f3d46b258a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_71d9196cabd74de2aa89344d641e5bba",
            "placeholder": "​",
            "style": "IPY_MODEL_c65550c07c0c42b0a983ff1f05721430",
            "value": "Generating train split: "
          }
        },
        "9d0f9a569f2844e7bd69790ad8b61ca7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a3aafb8172d5495687c527649272d1dc",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_61e046dd807a4243a6022ada12942a77",
            "value": 1
          }
        },
        "a3aafb8172d5495687c527649272d1dc": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        },
        "a9843f89c1054ec4adc226fdcdb1aba4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "aaa1bc8a499e41abae85b86ceb29b66b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d0d5d38ac0344244b0e688102ee8b13a",
            "placeholder": "​",
            "style": "IPY_MODEL_851855ecf5fb44f39d1e79162d5351f0",
            "value": " 8848/0 [00:00&lt;00:00, 135575.94 examples/s]"
          }
        },
        "afc6e86bf48d4bff96eea6567a82d447": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c41185e10467436abd9a100d0e797511": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c6216f0615814f92ba52a5da45e0f28d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f6edd1ad7deb4ac3b66e331117acefbb",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6b945880de87407fb4f279613b007654",
            "value": 1
          }
        },
        "c65550c07c0c42b0a983ff1f05721430": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "caf7fa7133774ce3877d1c1b1b729f66": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ef2c5cd0f4d84a68bf4ca6a88f40d9ed",
              "IPY_MODEL_9bcde6b4a56a467da9ea4056f36fbfbe",
              "IPY_MODEL_84f0d5fa529543ed84627e79ea0ef49d"
            ],
            "layout": "IPY_MODEL_1d5c753460984112ab52a18c1e470053"
          }
        },
        "d0d5d38ac0344244b0e688102ee8b13a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d1f38e16bca04d528d84a2a05ce56312": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dc63be992bf54c75bd49a19472723a89": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_103cf81c5cbd41d8a0051ad1fe6d151d",
            "placeholder": "​",
            "style": "IPY_MODEL_870238d863004b1ca038d51daae50e84",
            "value": " 6001888/6001888 [06:35&lt;00:00, 13168.05 examples/s]"
          }
        },
        "df5f7850ad134cc2905faf9af63074c0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e29ce04398f1496c856062c355a4555d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_85edd6b782084728bc5ca6512b8cd7a8",
            "placeholder": "​",
            "style": "IPY_MODEL_3b805a8f92e84d739357e8cc54f4deba",
            "value": "Map (num_proc=24): 100%"
          }
        },
        "ef2c5cd0f4d84a68bf4ca6a88f40d9ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7a5e77e55f7941e6a59d26f4f2371c03",
            "placeholder": "​",
            "style": "IPY_MODEL_13471ff80cdc4ef18e360b8534afe0b7",
            "value": "Map (num_proc=24): 100%"
          }
        },
        "f166787b613f4a4ab0f7b1e38e3e153e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_22f09c483a434a35832f1a0a0e20a2a9",
            "max": 6001888,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_10d88eb79fcd4fffa27d3f40ebe8ac23",
            "value": 6001888
          }
        },
        "f6edd1ad7deb4ac3b66e331117acefbb": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "20px"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}